{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/lightgbm/__init__.py:48: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.3) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.3</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.27</td>\n",
       "      <td>13.9</td>\n",
       "      <td>0.057</td>\n",
       "      <td>45</td>\n",
       "      <td>155</td>\n",
       "      <td>0.99807</td>\n",
       "      <td>2.94</td>\n",
       "      <td>0.41</td>\n",
       "      <td>8.8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.49</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0.065</td>\n",
       "      <td>17</td>\n",
       "      <td>143</td>\n",
       "      <td>0.9937</td>\n",
       "      <td>3.22</td>\n",
       "      <td>0.52</td>\n",
       "      <td>9.2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.1</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.35</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.04</td>\n",
       "      <td>60</td>\n",
       "      <td>171</td>\n",
       "      <td>0.999</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.59</td>\n",
       "      <td>9.1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.47</td>\n",
       "      <td>16.9</td>\n",
       "      <td>0.052</td>\n",
       "      <td>51</td>\n",
       "      <td>188</td>\n",
       "      <td>0.99944</td>\n",
       "      <td>3.09</td>\n",
       "      <td>0.62</td>\n",
       "      <td>9.3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.38</td>\n",
       "      <td>15.3</td>\n",
       "      <td>0.045</td>\n",
       "      <td>54</td>\n",
       "      <td>120</td>\n",
       "      <td>0.9975</td>\n",
       "      <td>3.18</td>\n",
       "      <td>0.42</td>\n",
       "      <td>9.8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  fixed acidity volatile acidity citric acid residual sugar chlorides  \\\n",
       "0           7.3             0.19        0.27           13.9     0.057   \n",
       "1           6.2              0.2        0.49            1.6     0.065   \n",
       "2           7.1             0.23        0.35           16.5      0.04   \n",
       "3           7.5              0.2        0.47           16.9     0.052   \n",
       "4             7             0.15        0.38           15.3     0.045   \n",
       "\n",
       "  free sulfur dioxide total sulfur dioxide  density    pH sulphates alcohol  \\\n",
       "0                  45                  155  0.99807  2.94      0.41     8.8   \n",
       "1                  17                  143   0.9937  3.22      0.52     9.2   \n",
       "2                  60                  171    0.999  3.16      0.59     9.1   \n",
       "3                  51                  188  0.99944  3.09      0.62     9.3   \n",
       "4                  54                  120   0.9975  3.18      0.42     9.8   \n",
       "\n",
       "  quality  \n",
       "0       8  \n",
       "1       6  \n",
       "2       6  \n",
       "3       5  \n",
       "4       6  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../data/train.csv', sep=';', header=None, \n",
    "                    names=['fixed acidity','volatile acidity','citric acid',\n",
    "                           'residual sugar','chlorides','free sulfur dioxide','total sulfur dioxide',\n",
    "                           'density','pH','sulphates','alcohol','quality'])\n",
    "columns = train.columns.tolist()\n",
    "columns.remove('quality')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6          1535\n",
      "5          1024\n",
      "7           593\n",
      "8           125\n",
      "4           104\n",
      "3            13\n",
      "9             3\n",
      "quality       1\n",
      "Name: quality, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>fixed acidity</td>\n",
       "      <td>volatile acidity</td>\n",
       "      <td>citric acid</td>\n",
       "      <td>residual sugar</td>\n",
       "      <td>chlorides</td>\n",
       "      <td>free sulfur dioxide</td>\n",
       "      <td>total sulfur dioxide</td>\n",
       "      <td>density</td>\n",
       "      <td>pH</td>\n",
       "      <td>sulphates</td>\n",
       "      <td>alcohol</td>\n",
       "      <td>quality</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "629  fixed acidity  volatile acidity  citric acid  residual sugar  chlorides   \n",
       "\n",
       "     free sulfur dioxide  total sulfur dioxide  density  pH  sulphates  \\\n",
       "629  free sulfur dioxide  total sulfur dioxide  density  pH  sulphates   \n",
       "\n",
       "     alcohol  quality  \n",
       "629  alcohol  quality  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train['quality'].value_counts())\n",
    "train[train['quality'] == 'quality']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop(train.index[629], inplace=True)\n",
    "train['quality'] = train['quality'].astype(int) - 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.9</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.35</td>\n",
       "      <td>4.60</td>\n",
       "      <td>0.032</td>\n",
       "      <td>32.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>0.99458</td>\n",
       "      <td>3.15</td>\n",
       "      <td>0.45</td>\n",
       "      <td>11.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.9</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.32</td>\n",
       "      <td>1.20</td>\n",
       "      <td>0.024</td>\n",
       "      <td>20.0</td>\n",
       "      <td>166.0</td>\n",
       "      <td>0.99208</td>\n",
       "      <td>3.05</td>\n",
       "      <td>0.54</td>\n",
       "      <td>9.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.3</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.29</td>\n",
       "      <td>2.00</td>\n",
       "      <td>0.022</td>\n",
       "      <td>33.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.98902</td>\n",
       "      <td>3.04</td>\n",
       "      <td>0.54</td>\n",
       "      <td>12.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.4</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.021</td>\n",
       "      <td>52.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>0.99038</td>\n",
       "      <td>2.93</td>\n",
       "      <td>0.32</td>\n",
       "      <td>11.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.24</td>\n",
       "      <td>1.80</td>\n",
       "      <td>0.047</td>\n",
       "      <td>29.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>0.99251</td>\n",
       "      <td>3.30</td>\n",
       "      <td>0.43</td>\n",
       "      <td>9.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            8.9              0.30         0.35            4.60      0.032   \n",
       "1            6.9              0.31         0.32            1.20      0.024   \n",
       "2            6.3              0.19         0.29            2.00      0.022   \n",
       "3            8.4              0.31         0.31            0.95      0.021   \n",
       "4            7.0              0.24         0.24            1.80      0.047   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 32.0                 148.0  0.99458  3.15       0.45   \n",
       "1                 20.0                 166.0  0.99208  3.05       0.54   \n",
       "2                 33.0                  96.0  0.98902  3.04       0.54   \n",
       "3                 52.0                 148.0  0.99038  2.93       0.32   \n",
       "4                 29.0                  91.0  0.99251  3.30       0.43   \n",
       "\n",
       "   alcohol  \n",
       "0     11.5  \n",
       "1      9.8  \n",
       "2     12.8  \n",
       "3     11.5  \n",
       "4      9.9  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('../data/test.csv', sep=';', header=None, \n",
    "                    names=['fixed acidity','volatile acidity','citric acid',\n",
    "                           'residual sugar','chlorides','free sulfur dioxide','total sulfur dioxide',\n",
    "                           'density','pH','sulphates','alcohol'])\n",
    "data = pd.concat([train, test], ignore_index=True)\n",
    "data = data.astype(float)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_labels:7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3    1535\n",
       "2    1024\n",
       "4     593\n",
       "5     125\n",
       "1     104\n",
       "0      13\n",
       "6       3\n",
       "Name: quality, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels = len(set(train['quality'].values))\n",
    "print('num_labels:{}'.format(num_labels))\n",
    "train['quality'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0 [10, 20, 30, 40, 50, 60] 289.0\n",
      "[2.0, 10, 20, 30, 40, 50, 60, 289.0]\n"
     ]
    }
   ],
   "source": [
    "def binning(col, cut_points, labels=None):\n",
    "\n",
    "    #Define min and max values:\n",
    "\n",
    "    minval = col.min()\n",
    "\n",
    "    maxval = col.max()\n",
    "\n",
    "    #利用最大值和最小值创建分箱点的列表\n",
    "    print(minval, cut_points, maxval)\n",
    "    break_points = [minval] + cut_points + [maxval]\n",
    "    print(break_points)\n",
    "    #如果没有标签，则使用默认标签0 ... (n-1)\n",
    "\n",
    "    if not labels:\n",
    "\n",
    "        labels = range(len(cut_points)+1)\n",
    "\n",
    "    #使用pandas的cut功能分箱\n",
    "\n",
    "    colBin = pd.cut(col,bins=break_points,labels=labels,include_lowest=True)\n",
    "\n",
    "    return colBin\n",
    "\n",
    "#为年龄分箱:\n",
    "\n",
    "cut_points = [10, 20, 30, 40, 50, 60]\n",
    "\n",
    "labels = [\"1111\",\"2222\",\"3333\", \"4444\", \"5555\", '6666', '7']\n",
    "\n",
    "data[\"free_sulfur_dioxide_bin\"] = binning(data[\"free sulfur dioxide\"], cut_points, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "object_columns = ['free_sulfur_dioxide_bin']\n",
    "for col in object_columns:\n",
    "    data = pd.concat([data, pd.get_dummies(data[col], prefix=col+'_')], axis=1)\n",
    "    data.drop(col, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = data.columns.tolist()\n",
    "columns.remove('density')\n",
    "columns.remove('free sulfur dioxide')\n",
    "columns.remove('quality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def data_expand(data, label, num):\n",
    "#     tmp = data[data['quality'] == label]\n",
    "#     for i in range(num):\n",
    "#         data = data.append(tmp)\n",
    "#     return data\n",
    "# train = data_expand(train, 0, 10)\n",
    "# train = data_expand(train, 1, 5)\n",
    "# train = data_expand(train, 5, 3)\n",
    "# train = data_expand(train, 6, 30)\n",
    "# train['quality'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_expand = train.append(train[train['quality'] == 6])\n",
    "# x_train = train_expand[columns].astype(float)\n",
    "# y_train = train_expand['quality']\n",
    "# from imblearn.over_sampling import SMOTE\n",
    "# from collections import Counter\n",
    "# smo = SMOTE(random_state=0)\n",
    "# x_smo, y_smo = smo.fit_sample(x_train, y_train)\n",
    "# print(Counter(y_smo))\n",
    "\n",
    "# y_smo = y_smo.reshape(-1, 1)\n",
    "# train = np.concatenate((y_smo, x_smo), axis=1)\n",
    "# train = pd.DataFrame(train, columns=['quality'] + columns)\n",
    "# train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = data[data['quality'].notnull()]\n",
    "test = data[data['quality'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.708649\tvalid_1's multi_logloss: 1.18775\n",
      "[200]\ttraining's multi_logloss: 0.486178\tvalid_1's multi_logloss: 1.14338\n",
      "[300]\ttraining's multi_logloss: 0.353185\tvalid_1's multi_logloss: 1.13421\n",
      "[400]\ttraining's multi_logloss: 0.266242\tvalid_1's multi_logloss: 1.13881\n",
      "Early stopping, best iteration is:\n",
      "[338]\ttraining's multi_logloss: 0.316226\tvalid_1's multi_logloss: 1.13385\n",
      "fold 2\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.734948\tvalid_1's multi_logloss: 0.958642\n",
      "[200]\ttraining's multi_logloss: 0.507582\tvalid_1's multi_logloss: 0.906696\n",
      "[300]\ttraining's multi_logloss: 0.372489\tvalid_1's multi_logloss: 0.887596\n",
      "[400]\ttraining's multi_logloss: 0.284076\tvalid_1's multi_logloss: 0.88514\n",
      "[500]\ttraining's multi_logloss: 0.224992\tvalid_1's multi_logloss: 0.888516\n",
      "Early stopping, best iteration is:\n",
      "[417]\ttraining's multi_logloss: 0.272055\tvalid_1's multi_logloss: 0.883555\n",
      "fold 3\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.728551\tvalid_1's multi_logloss: 0.96412\n",
      "[200]\ttraining's multi_logloss: 0.499449\tvalid_1's multi_logloss: 0.919899\n",
      "[300]\ttraining's multi_logloss: 0.363964\tvalid_1's multi_logloss: 0.911942\n",
      "[400]\ttraining's multi_logloss: 0.276716\tvalid_1's multi_logloss: 0.919463\n",
      "Early stopping, best iteration is:\n",
      "[305]\ttraining's multi_logloss: 0.358702\tvalid_1's multi_logloss: 0.911017\n",
      "fold 4\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.728882\tvalid_1's multi_logloss: 0.982607\n",
      "[200]\ttraining's multi_logloss: 0.500933\tvalid_1's multi_logloss: 0.938746\n",
      "[300]\ttraining's multi_logloss: 0.366023\tvalid_1's multi_logloss: 0.932379\n",
      "[400]\ttraining's multi_logloss: 0.279169\tvalid_1's multi_logloss: 0.934869\n",
      "Early stopping, best iteration is:\n",
      "[331]\ttraining's multi_logloss: 0.33527\tvalid_1's multi_logloss: 0.930228\n",
      "fold 5\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.72296\tvalid_1's multi_logloss: 1.02843\n",
      "[200]\ttraining's multi_logloss: 0.496204\tvalid_1's multi_logloss: 0.99436\n",
      "[300]\ttraining's multi_logloss: 0.361912\tvalid_1's multi_logloss: 0.994708\n",
      "Early stopping, best iteration is:\n",
      "[219]\ttraining's multi_logloss: 0.46527\tvalid_1's multi_logloss: 0.991726\n"
     ]
    }
   ],
   "source": [
    "x_train = train[columns].astype(float)\n",
    "y_train = train['quality']\n",
    "x_test = test[columns]\n",
    "\n",
    "params = {\n",
    "    'num_leaves': 60,\n",
    "    'min_data_in_leaf': 30,\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': num_labels,\n",
    "    'max_depth': -1,\n",
    "    'learning_rate': 0.03,\n",
    "    \"min_sum_hessian_in_leaf\": 6,\n",
    "    \"boosting\": \"gbdt\",\n",
    "    \"feature_fraction\": 0.9,\n",
    "    \"bagging_freq\": 1,\n",
    "    \"bagging_fraction\": 0.8,\n",
    "    \"bagging_seed\": 11,\n",
    "    \"lambda_l1\": 0.1,\n",
    "    \"verbosity\": -1,\n",
    "    \"nthread\": 15,\n",
    "    'metric': 'multi_logloss',\n",
    "    \"random_state\": 0}\n",
    "\n",
    "\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "oof = np.zeros((x_train.shape[0], num_labels))\n",
    "preds_prob = np.zeros((x_test.shape[0], num_labels))\n",
    "\n",
    "## train and predict\n",
    "feature_importance_df = pd.DataFrame()\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(x_train)):\n",
    "    print(\"fold {}\".format(fold_ + 1))\n",
    "    trn_data = lgb.Dataset(x_train.iloc[trn_idx], label=y_train.iloc[trn_idx])\n",
    "    val_data = lgb.Dataset(x_train.iloc[val_idx], label=y_train.iloc[val_idx])\n",
    "    clf = lgb.train(\n",
    "        params,\n",
    "        trn_data,\n",
    "        valid_sets=[trn_data, val_data],\n",
    "        num_boost_round = 10000,\n",
    "        verbose_eval = 100,\n",
    "        early_stopping_rounds = 100)\n",
    "    oof[val_idx] = clf.predict(x_train.iloc[val_idx], num_iteration=clf.best_iteration)\n",
    "\n",
    "    preds_prob += clf.predict(x_test, num_iteration=clf.best_iteration) / folds.n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6302619958787166"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "oof = np.argmax(oof, axis=1)\n",
    "accuracy_score(y_train, oof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  pred\n",
       "0   1     7\n",
       "1   2     6\n",
       "2   3     6\n",
       "3   4     5\n",
       "4   5     5"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_prob1 = preds_prob\n",
    "preds = np.argmax(preds_prob, axis=1)\n",
    "submission = pd.DataFrame({'id': range(len(preds)), 'pred': preds+3})\n",
    "submission['id'] = submission['id'] + 1\n",
    "submission.to_csv(\"../data/lgb_submission.csv\", index=False, header=False)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import xgboost as xgb\n",
    "# x_train = train[columns].astype(float)\n",
    "# y_train = train['quality']\n",
    "# x_test = test[columns]\n",
    "\n",
    "# params = {\n",
    "#     'booster': 'gbtree',\n",
    "#     'objective': 'multi:softprob',\n",
    "#     'num_class': num_labels,\n",
    "#     'eval_metric': 'mlogloss',\n",
    "#     'gamma': 0.1,\n",
    "#     'max_depth': 8,\n",
    "#     'alpha': 0,\n",
    "#     'lambda': 0,\n",
    "#     'subsample': 0.7,\n",
    "#     'colsample_bytree': 0.5,\n",
    "#     'min_child_weight': 3,\n",
    "#     'silent': 0,\n",
    "#     'eta': 0.03,\n",
    "#     'nthread': -1,\n",
    "#     'missing': 1,\n",
    "#     'seed': 0,\n",
    "# }\n",
    "\n",
    "# folds = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "# oof_prob = np.zeros((x_train.shape[0], num_labels))\n",
    "# preds_prob = np.zeros((x_test.shape[0], num_labels))\n",
    "\n",
    "# num_round = 10000\n",
    "# ## train and predict\n",
    "# feature_importance_df = pd.DataFrame()\n",
    "# for fold_, (trn_idx, val_idx) in enumerate(folds.split(train)):\n",
    "#     print(\"fold {}\".format(fold_ + 1))\n",
    "#     trn_data = xgb.DMatrix(x_train.iloc[trn_idx], label=y_train.iloc[trn_idx])\n",
    "#     val_data = xgb.DMatrix(x_train.iloc[val_idx], label=y_train.iloc[val_idx])\n",
    "\n",
    "#     watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n",
    "#     clf = xgb.train(params, \n",
    "#                     trn_data, \n",
    "#                     num_round, \n",
    "#                     watchlist, \n",
    "#                     verbose_eval=100, \n",
    "#                     early_stopping_rounds=100)\n",
    "# #     fold_importance_df = pd.DataFrame()\n",
    "# #     fold_importance_df[\"Feature\"] = clf.get_fscore().keys()\n",
    "# #     fold_importance_df[\"importance\"] = clf.get_fscore().values()\n",
    "# #     fold_importance_df[\"fold\"] = fold_ + 1\n",
    "# #     feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "    \n",
    "#     oof_prob[val_idx] = clf.predict(xgb.DMatrix(x_train.iloc[val_idx]), ntree_limit=clf.best_ntree_limit)\n",
    "#     preds_prob += clf.predict(xgb.DMatrix(test), ntree_limit=clf.best_ntree_limit) / folds.n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.metrics import accuracy_score\n",
    "# oof = np.argmax(oof_prob, axis=1)\n",
    "# accuracy_score(y_train, oof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preds_prob2 = preds_prob\n",
    "# preds = np.argmax(preds_prob, axis=1)\n",
    "# submission = pd.DataFrame({'id': range(len(preds)), 'pred': preds+3})\n",
    "# submission['id'] = submission['id'] + 1\n",
    "# submission.to_csv(\"../data/xgb_submission.csv\", index=False, header=False)\n",
    "# submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 直接相加，结果并不好\n",
    "# preds_prob = (preds_prob1+preds_prob1)/2\n",
    "# preds = np.argmax(preds_prob, axis=1)\n",
    "# submission = pd.DataFrame({'id': range(len(preds)), 'pred': preds+3})\n",
    "# submission['id'] = submission['id'] + 1\n",
    "# submission.to_csv(\"../data/merge_submission.csv\", index=False, header=False)\n",
    "# submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
